{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TwentyNewsGroup\n",
    "\n",
    "20 Newsgroup 数据集包含了约 20000 篇来自于不同的新闻组的文档，最早由 Ken Lang 搜集整理。本部分包含了对于数据集的抓取、特征提取、简单分类器训练、主题模型训练等。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 配置环境变量\n",
    "import sys\n",
    "sys.path.append('./')\n",
    "sys.path.append('../')\n",
    "\n",
    "# 引入外部的封装模块\n",
    "from twenty_news_group import TwentyNewsGroup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "数据集结构 -> dict_keys(['data', 'filenames', 'target_names', 'target', 'DESCR', 'description'])\n",
      "文档数目 -> 11314\n",
      "目标分类 -> ['sci.space', 'comp.sys.mac.hardware', 'sci.electronics', 'comp.sys.mac.hardware', 'sci.space', 'rec.sport.hockey', 'talk.religion.misc', 'sci.med', 'talk.religion.misc', 'talk.politics.guns']\n"
     ]
    }
   ],
   "source": [
    "# 实例化对象\n",
    "twp = TwentyNewsGroup()\n",
    "\n",
    "# 抓取数据\n",
    "twp.fetch_data()\n",
    "\n",
    "twenty_train = twp.data['train']\n",
    "\n",
    "print(\"数据集结构\", \"->\", twenty_train.keys())\n",
    "\n",
    "print(\"文档数目\", \"->\", len(twenty_train.data))\n",
    "\n",
    "print(\"目标分类\", \"->\",[ twenty_train.target_names[t] for t in twenty_train.target[:10]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DTM 结构 -> (11314, 130107)\n",
      "词对应下标 -> 27366\n"
     ]
    }
   ],
   "source": [
    "# 进行特征提取\n",
    "\n",
    "# 构建文档-词矩阵（Document-Term Matrix）\n",
    "\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "\n",
    "count_vect = CountVectorizer()\n",
    "\n",
    "X_train_counts = count_vect.fit_transform(twenty_train.data)\n",
    "\n",
    "print(\"DTM 结构\",\"->\",X_train_counts.shape)\n",
    "\n",
    "# 查看某个词在词表中的下标\n",
    "print(\"词对应下标\",\"->\", count_vect.vocabulary_.get(u'algorithm'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "某文档 TF 特征向量 ->   (0, 6447)\t0.0380693493813\n",
      "  (0, 37842)\t0.0380693493813\n",
      "  (0, 128402)\t0.0380693493813\n",
      "  (0, 47566)\t0.0380693493813\n",
      "  (0, 110220)\t0.0380693493813\n",
      "  (0, 61997)\t0.0380693493813\n",
      "  (0, 109061)\t0.0380693493813\n",
      "  (0, 124596)\t0.152277397525\n",
      "  (0, 125288)\t0.0380693493813\n",
      "  (0, 104717)\t0.0380693493813\n",
      "  (0, 89919)\t0.0380693493813\n",
      "  (0, 123796)\t0.0380693493813\n",
      "  (0, 114508)\t0.0380693493813\n",
      "  (0, 124597)\t0.0380693493813\n",
      "  (0, 87265)\t0.0380693493813\n",
      "  (0, 74675)\t0.0380693493813\n",
      "  (0, 110321)\t0.0380693493813\n",
      "  (0, 106960)\t0.0380693493813\n",
      "  (0, 117211)\t0.0380693493813\n",
      "  (0, 104813)\t0.0380693493813\n",
      "  (0, 80638)\t0.0380693493813\n",
      "  (0, 71079)\t0.0380693493813\n",
      "  (0, 101151)\t0.0380693493813\n",
      "  (0, 74757)\t0.0380693493813\n",
      "  (0, 111533)\t0.0380693493813\n",
      "  :\t:\n",
      "  (11313, 58076)\t0.0129045687035\n",
      "  (11313, 113763)\t0.0129045687035\n",
      "  (11313, 125053)\t0.0129045687035\n",
      "  (11313, 109035)\t0.0516182748138\n",
      "  (11313, 128387)\t0.0516182748138\n",
      "  (11313, 86864)\t0.0516182748138\n",
      "  (11313, 40015)\t0.0258091374069\n",
      "  (11313, 3802)\t0.0258091374069\n",
      "  (11313, 2336)\t0.0258091374069\n",
      "  (11313, 72384)\t0.0129045687035\n",
      "  (11313, 74675)\t0.0258091374069\n",
      "  (11313, 28146)\t0.0258091374069\n",
      "  (11313, 56283)\t0.0258091374069\n",
      "  (11313, 6475)\t0.0258091374069\n",
      "  (11313, 28989)\t0.0129045687035\n",
      "  (11313, 66608)\t0.0129045687035\n",
      "  (11313, 64095)\t0.0129045687035\n",
      "  (11313, 95162)\t0.0129045687035\n",
      "  (11313, 87620)\t0.0129045687035\n",
      "  (11313, 76032)\t0.0129045687035\n",
      "  (11313, 90379)\t0.0129045687035\n",
      "  (11313, 47982)\t0.0129045687035\n",
      "  (11313, 111322)\t0.0129045687035\n",
      "  (11313, 50527)\t0.0516182748138\n",
      "  (11313, 56979)\t0.0129045687035\n",
      "某文档 TF-IDF 特征向量 ->   (0, 56979)\t0.00934246895801\n",
      "  (0, 110522)\t0.217743107845\n",
      "  (0, 115702)\t0.215666053557\n",
      "  (0, 117854)\t0.1899848004\n",
      "  (0, 50527)\t0.0532692788455\n",
      "  (0, 110524)\t0.145162071897\n",
      "  (0, 107374)\t0.145162071897\n",
      "  (0, 111322)\t0.00934246895801\n",
      "  (0, 99721)\t0.0128215131665\n",
      "  (0, 45383)\t0.291708457185\n",
      "  (0, 102856)\t0.0815078719627\n",
      "  (0, 101045)\t0.0448116221997\n",
      "  (0, 29573)\t0.0329514451724\n",
      "  (0, 21977)\t0.0900682896817\n",
      "  (0, 47982)\t0.0229602915862\n",
      "  (0, 105245)\t0.0432877785681\n",
      "  (0, 90379)\t0.0194380640015\n",
      "  (0, 75786)\t0.222425200646\n",
      "  (0, 89076)\t0.121550064603\n",
      "  (0, 117838)\t0.0771168776463\n",
      "  (0, 76032)\t0.00937306884829\n",
      "  (0, 10368)\t0.0398715691717\n",
      "  (0, 87620)\t0.0173966789945\n",
      "  (0, 95162)\t0.016811221808\n",
      "  (0, 64095)\t0.0172742995454\n",
      "  :\t:\n",
      "  (11313, 39887)\t0.0338197456131\n",
      "  (11313, 100236)\t0.0353400566223\n",
      "  (11313, 62867)\t0.0358465799842\n",
      "  (11313, 103116)\t0.0358465799842\n",
      "  (11313, 79491)\t0.0380223897695\n",
      "  (11313, 117188)\t0.0355034195389\n",
      "  (11313, 37831)\t0.0368203246304\n",
      "  (11313, 74433)\t0.0209335906129\n",
      "  (11313, 12346)\t0.0188792300701\n",
      "  (11313, 18349)\t0.0178360697807\n",
      "  (11313, 15400)\t0.0431603195329\n",
      "  (11313, 19227)\t0.0612519453434\n",
      "  (11313, 9008)\t0.0724285441192\n",
      "  (11313, 92612)\t0.0403869132175\n",
      "  (11313, 40451)\t0.0209335906129\n",
      "  (11313, 70881)\t0.0418671812257\n",
      "  (11313, 70879)\t0.0212366168983\n",
      "  (11313, 83575)\t0.0215801597664\n",
      "  (11313, 99224)\t0.0199875267459\n",
      "  (11313, 99107)\t0.019297350619\n",
      "  (11313, 70880)\t0.0219767509023\n",
      "  (11313, 95327)\t0.0215801597664\n",
      "  (11313, 30123)\t0.0408346302289\n",
      "  (11313, 27601)\t0.0215801597664\n",
      "  (11313, 93537)\t0.0248032054854\n"
     ]
    }
   ],
   "source": [
    "# 构建文档的 TF 特征向量\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "\n",
    "tf_transformer = TfidfTransformer(use_idf=False).fit(X_train_counts)\n",
    "X_train_tf = tf_transformer.transform(X_train_counts)\n",
    "\n",
    "print(\"某文档 TF 特征向量\",\"->\",X_train_tf)\n",
    "\n",
    "# 构建文档的 TF-IDF 特征向量\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "\n",
    "tf_transformer = TfidfTransformer().fit(X_train_counts)\n",
    "X_train_tfidf = tf_transformer.transform(X_train_counts)\n",
    "\n",
    "print(\"某文档 TF-IDF 特征向量\",\"->\",X_train_tfidf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 训练分类器\n",
    "twp.train_classifier()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'God is love' => soc.religion.christian\n",
      "'OpenGL on the GPU is fast' => rec.autos\n",
      "                          precision    recall  f1-score   support\n",
      "\n",
      "             alt.atheism       0.79      0.50      0.61       319\n",
      "           comp.graphics       0.75      0.71      0.73       389\n",
      " comp.os.ms-windows.misc       0.78      0.71      0.74       394\n",
      "comp.sys.ibm.pc.hardware       0.65      0.80      0.72       392\n",
      "   comp.sys.mac.hardware       0.88      0.77      0.82       385\n",
      "          comp.windows.x       0.86      0.77      0.81       395\n",
      "            misc.forsale       0.89      0.83      0.86       390\n",
      "               rec.autos       0.84      0.92      0.88       396\n",
      "         rec.motorcycles       0.87      0.96      0.92       398\n",
      "      rec.sport.baseball       0.89      0.93      0.91       397\n",
      "        rec.sport.hockey       0.87      0.97      0.92       399\n",
      "               sci.crypt       0.73      0.95      0.83       396\n",
      "         sci.electronics       0.81      0.65      0.72       393\n",
      "                 sci.med       0.87      0.79      0.83       396\n",
      "               sci.space       0.83      0.92      0.88       394\n",
      "  soc.religion.christian       0.52      0.96      0.68       398\n",
      "      talk.politics.guns       0.66      0.90      0.76       364\n",
      "   talk.politics.mideast       0.94      0.88      0.91       376\n",
      "      talk.politics.misc       0.97      0.35      0.52       310\n",
      "      talk.religion.misc       1.00      0.08      0.15       251\n",
      "\n",
      "             avg / total       0.82      0.79      0.77      7532\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[158,   0,   1,   1,   0,   1,   0,   3,   7,   1,   2,   6,   1,\n",
       "          8,   3, 114,   6,   7,   0,   0],\n",
       "       [  1, 278,  11,  15,   9,  21,   1,   5,   1,   7,   2,  16,   8,\n",
       "          2,   7,   3,   1,   1,   0,   0],\n",
       "       [  0,  17, 280,  44,   2,   9,   0,   4,   5,   5,   2,  15,   1,\n",
       "          0,   5,   5,   0,   0,   0,   0],\n",
       "       [  0,   6,  18, 312,  12,   3,   8,   6,   1,   0,   1,   3,  17,\n",
       "          0,   5,   0,   0,   0,   0,   0],\n",
       "       [  0,   3,  13,  32, 295,   2,   8,   5,   2,   4,   1,   6,   9,\n",
       "          1,   3,   0,   1,   0,   0,   0],\n",
       "       [  1,  30,  23,  11,   1, 306,   2,   1,   2,   2,   0,   8,   0,\n",
       "          2,   4,   1,   1,   0,   0,   0],\n",
       "       [  0,   4,   4,  19,   7,   1, 322,   8,   2,   3,   6,   1,   7,\n",
       "          3,   2,   1,   0,   0,   0,   0],\n",
       "       [  0,   1,   1,   2,   0,   1,   6, 365,   4,   2,   2,   0,   5,\n",
       "          1,   3,   0,   2,   0,   1,   0],\n",
       "       [  0,   0,   0,   1,   0,   0,   2,   9, 384,   0,   0,   1,   0,\n",
       "          0,   0,   1,   0,   0,   0,   0],\n",
       "       [  0,   0,   0,   0,   1,   0,   2,   3,   0, 368,  19,   0,   0,\n",
       "          0,   1,   2,   1,   0,   0,   0],\n",
       "       [  0,   0,   0,   1,   0,   0,   0,   1,   0,   5, 388,   0,   0,\n",
       "          0,   1,   3,   0,   0,   0,   0],\n",
       "       [  0,   3,   2,   0,   1,   2,   2,   2,   0,   2,   0, 377,   1,\n",
       "          1,   1,   1,   1,   0,   0,   0],\n",
       "       [  1,   9,   4,  36,   5,   2,   3,   7,   8,   2,   2,  37, 255,\n",
       "          5,  12,   4,   0,   1,   0,   0],\n",
       "       [  2,   6,   1,   3,   2,   2,   3,   2,  10,   4,   5,   4,   9,\n",
       "        314,   3,  22,   2,   2,   0,   0],\n",
       "       [  0,   6,   0,   0,   0,   4,   0,   3,   1,   0,   2,   1,   1,\n",
       "          5, 363,   6,   2,   0,   0,   0],\n",
       "       [  2,   2,   2,   1,   0,   0,   0,   0,   2,   1,   1,   0,   0,\n",
       "          2,   3, 382,   0,   0,   0,   0],\n",
       "       [  0,   0,   0,   1,   0,   0,   2,   4,   3,   3,   1,   9,   0,\n",
       "          3,   2,   6, 329,   1,   0,   0],\n",
       "       [  0,   1,   0,   0,   0,   3,   0,   1,   2,   1,   3,   4,   1,\n",
       "          0,   1,  23,   7, 329,   0,   0],\n",
       "       [  0,   1,   0,   0,   0,   0,   1,   3,   5,   1,   4,  22,   1,\n",
       "          6,  11,  27, 115,   4, 109,   0],\n",
       "       [ 35,   3,   1,   0,   0,   0,   1,   4,   1,   1,   6,   3,   0,\n",
       "          6,   5, 127,  30,   5,   2,  21]])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 执行预测\n",
    "docs_new = ['God is love', 'OpenGL on the GPU is fast']\n",
    "predicted = twp.predict(docs_new)\n",
    "\n",
    "for doc, category in zip(docs_new, predicted):\n",
    "    print('%r => %s' % (doc, twenty_train.target_names[category]))\n",
    "    \n",
    "# 执行模型评测\n",
    "twp.fetch_data(subset='test')\n",
    "\n",
    "predicted = twp.predict(twp.data['test'].data)\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "# 误差计算\n",
    "\n",
    "# 简单误差均值\n",
    "np.mean(predicted == twp.data['test'].target)   \n",
    "\n",
    "# Metrics\n",
    "\n",
    "from sklearn import metrics\n",
    "\n",
    "print(metrics.classification_report(\n",
    "    twp.data['test'].target, predicted,\n",
    "    target_names=twp.data['test'].target_names))\n",
    "\n",
    "# Confusion Matrix\n",
    "metrics.confusion_matrix(twp.data['test'].target, predicted)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Topic 0 : stream s1 astronaut zoo laurentian maynard s2 gtoal pem fpu\n",
      "Topic 1 : 145 cx 0d bh sl 75u 6um m6 sy gld\n",
      "Topic 2 : apartment wpi mars nazis monash palestine ottoman sas winner gerard\n",
      "Topic 3 : livesey contest satellite tamu mathew orbital wpd marriage solntze pope\n",
      "Topic 4 : x11 contest lib font string contrib visual xterm ahl brake\n",
      "Topic 5 : ax g9v b8f a86 1d9 pl 0t wm 34u giz\n",
      "Topic 6 : printf null char manes behanna senate handgun civilians homicides magpie\n",
      "Topic 7 : buf jpeg chi tor bos det que uwo pit blah\n",
      "Topic 8 : oracle di t4 risc nist instruction msg postscript dma convex\n",
      "Topic 9 : candida cray yeast viking dog venus bloom symptoms observatory roby\n",
      "Topic 10 : cx ck hz lk mv cramer adl optilink k8 uw\n",
      "Topic 11 : ripem rsa sandvik w0 bosnia psuvm hudson utk defensive veal\n",
      "Topic 12 : db espn sabbath br widgets liar davidian urartu sdpa cooling\n",
      "Topic 13 : ripem dyer ucsu carleton adaptec tires chem alchemy lockheed rsa\n",
      "Topic 14 : ingr sv alomar jupiter borland het intergraph factory paradox captain\n",
      "Topic 15 : militia palestinian cpr pts handheld sharks igc apc jake lehigh\n",
      "Topic 16 : alaska duke col russia uoknor aurora princeton nsmca gene stereo\n",
      "Topic 17 : uuencode msg helmet eos satan dseg homosexual ics gear pyron\n",
      "Topic 18 : entries myers x11r4 radar remark cipher maine hamburg senior bontchev\n",
      "Topic 19 : cubs ufl vitamin temple gsfc mccall astro bellcore uranium wesleyan\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[(0,\n",
       "  [('stream', 0.0077211017660912575),\n",
       "   ('s1', 0.0051827365319261492),\n",
       "   ('astronaut', 0.0050496371416436066),\n",
       "   ('zoo', 0.0047744390567267493),\n",
       "   ('laurentian', 0.0043804887823957984),\n",
       "   ('maynard', 0.0043653680341875188),\n",
       "   ('s2', 0.0041050479793097476),\n",
       "   ('gtoal', 0.0041026151529143327),\n",
       "   ('pem', 0.0039248670974374065),\n",
       "   ('fpu', 0.0038304011470178261)]),\n",
       " (1,\n",
       "  [('145', 0.02684621934751372),\n",
       "   ('cx', 0.015146954155967162),\n",
       "   ('0d', 0.01120484561796793),\n",
       "   ('bh', 0.0072231553690531814),\n",
       "   ('sl', 0.0059666667248342306),\n",
       "   ('75u', 0.0054369260017737078),\n",
       "   ('6um', 0.0051805441512121509),\n",
       "   ('m6', 0.0047949629236150784),\n",
       "   ('sy', 0.0047903110764948124),\n",
       "   ('gld', 0.0047379217243407297)]),\n",
       " (2,\n",
       "  [('apartment', 0.0072609642043499501),\n",
       "   ('wpi', 0.0053815745581271305),\n",
       "   ('mars', 0.0043967649874930084),\n",
       "   ('nazis', 0.0037905780291177587),\n",
       "   ('monash', 0.0037466535736985556),\n",
       "   ('palestine', 0.0035695717910101754),\n",
       "   ('ottoman', 0.0031380911092148108),\n",
       "   ('sas', 0.0030500519912145005),\n",
       "   ('winner', 0.0029292079532012879),\n",
       "   ('gerard', 0.0027410209802652403)]),\n",
       " (3,\n",
       "  [('livesey', 0.0050832036603551652),\n",
       "   ('contest', 0.0044207000282105802),\n",
       "   ('satellite', 0.0039395199047106751),\n",
       "   ('tamu', 0.0037797816528796204),\n",
       "   ('mathew', 0.0037732170363337348),\n",
       "   ('orbital', 0.0036127495292406593),\n",
       "   ('wpd', 0.0036100478328117218),\n",
       "   ('marriage', 0.0035138972266393338),\n",
       "   ('solntze', 0.0034494462657597424),\n",
       "   ('pope', 0.0034233376599265707)]),\n",
       " (4,\n",
       "  [('x11', 0.008016782753309179),\n",
       "   ('contest', 0.0072335359299760321),\n",
       "   ('lib', 0.0068643393907931172),\n",
       "   ('font', 0.0068571696856338559),\n",
       "   ('string', 0.0067031167865392862),\n",
       "   ('contrib', 0.0060767485844960643),\n",
       "   ('visual', 0.0056764169856575366),\n",
       "   ('xterm', 0.0054449141461708258),\n",
       "   ('ahl', 0.0049228627875933803),\n",
       "   ('brake', 0.0049165296839170638)]),\n",
       " (5,\n",
       "  [('ax', 0.71578954742711209),\n",
       "   ('g9v', 0.030824025919576083),\n",
       "   ('b8f', 0.019324606462565567),\n",
       "   ('a86', 0.015706998305951757),\n",
       "   ('1d9', 0.012089342658528962),\n",
       "   ('pl', 0.0094699378046490847),\n",
       "   ('0t', 0.007190340931791884),\n",
       "   ('wm', 0.0057984132649124042),\n",
       "   ('34u', 0.0056218736181826824),\n",
       "   ('giz', 0.0054814594035164392)]),\n",
       " (6,\n",
       "  [('printf', 0.0091339785902918823),\n",
       "   ('null', 0.0076356736097640482),\n",
       "   ('char', 0.005103018776319055),\n",
       "   ('manes', 0.0046515852561957666),\n",
       "   ('behanna', 0.0041959118634166282),\n",
       "   ('senate', 0.0041802236233516709),\n",
       "   ('handgun', 0.0040628886430151921),\n",
       "   ('civilians', 0.0037724553817845667),\n",
       "   ('homicides', 0.0033007893523041456),\n",
       "   ('magpie', 0.0032897873298717954)]),\n",
       " (7,\n",
       "  [('buf', 0.010464316708879441),\n",
       "   ('jpeg', 0.0085778013654799253),\n",
       "   ('chi', 0.0063679805458149727),\n",
       "   ('tor', 0.0062874493177449489),\n",
       "   ('bos', 0.0062832356467473083),\n",
       "   ('det', 0.0062730967923696557),\n",
       "   ('que', 0.0053138399378702414),\n",
       "   ('uwo', 0.0052611578701104011),\n",
       "   ('pit', 0.0048502933608843357),\n",
       "   ('blah', 0.0044470956022154137)]),\n",
       " (8,\n",
       "  [('oracle', 0.0052687266537200629),\n",
       "   ('di', 0.0048713404976921861),\n",
       "   ('t4', 0.0048012922642039916),\n",
       "   ('risc', 0.0042667566748831789),\n",
       "   ('nist', 0.0035820105642170295),\n",
       "   ('instruction', 0.0035026170737589095),\n",
       "   ('msg', 0.0034896883663333429),\n",
       "   ('postscript', 0.0034487411282631813),\n",
       "   ('dma', 0.0034182312943306552),\n",
       "   ('convex', 0.0034059482331887038)]),\n",
       " (9,\n",
       "  [('candida', 0.0090121617750002821),\n",
       "   ('cray', 0.0061888971378692252),\n",
       "   ('yeast', 0.0059042076433987473),\n",
       "   ('viking', 0.0052290532759609841),\n",
       "   ('dog', 0.004338496469410205),\n",
       "   ('venus', 0.0043276010967435033),\n",
       "   ('bloom', 0.0041676362486736821),\n",
       "   ('symptoms', 0.0040976899419427936),\n",
       "   ('observatory', 0.0040238817513417145),\n",
       "   ('roby', 0.0038203351742483131)]),\n",
       " (10,\n",
       "  [('cx', 0.014758706716259064),\n",
       "   ('ck', 0.012491592948907462),\n",
       "   ('hz', 0.010568033927699092),\n",
       "   ('lk', 0.010048272072000907),\n",
       "   ('mv', 0.009913979014349943),\n",
       "   ('cramer', 0.0094595165423076531),\n",
       "   ('adl', 0.0089773226829855581),\n",
       "   ('optilink', 0.0084368584024709301),\n",
       "   ('k8', 0.0082946343769389301),\n",
       "   ('uw', 0.0079341454560415796)]),\n",
       " (11,\n",
       "  [('ripem', 0.0087821347426597467),\n",
       "   ('rsa', 0.0070042849745477007),\n",
       "   ('sandvik', 0.006917167945355236),\n",
       "   ('w0', 0.0045336433153537435),\n",
       "   ('bosnia', 0.0042685707643230651),\n",
       "   ('psuvm', 0.0036697962523835523),\n",
       "   ('hudson', 0.0035786123774222038),\n",
       "   ('utk', 0.0033831584058346773),\n",
       "   ('defensive', 0.0032251080743140271),\n",
       "   ('veal', 0.0030694146824802406)]),\n",
       " (12,\n",
       "  [('db', 0.037190362284272256),\n",
       "   ('espn', 0.0074632741247679006),\n",
       "   ('sabbath', 0.0065089747350081575),\n",
       "   ('br', 0.0056735709132596157),\n",
       "   ('widgets', 0.0052850384468319314),\n",
       "   ('liar', 0.0045418704234186699),\n",
       "   ('davidian', 0.0041427706402820405),\n",
       "   ('urartu', 0.0036827184121292372),\n",
       "   ('sdpa', 0.0034763031041864755),\n",
       "   ('cooling', 0.0034523478217537928)]),\n",
       " (13,\n",
       "  [('ripem', 0.0041493568379577603),\n",
       "   ('dyer', 0.0039677638558766066),\n",
       "   ('ucsu', 0.0039545633881215101),\n",
       "   ('carleton', 0.0037015748503318362),\n",
       "   ('adaptec', 0.0036007755736504219),\n",
       "   ('tires', 0.0034248509793288794),\n",
       "   ('chem', 0.0032982455761109667),\n",
       "   ('alchemy', 0.0030509780895570629),\n",
       "   ('lockheed', 0.0030444111576532307),\n",
       "   ('rsa', 0.0028461033213220835)]),\n",
       " (14,\n",
       "  [('ingr', 0.0057739123824076472),\n",
       "   ('sv', 0.0050134395779746089),\n",
       "   ('alomar', 0.0046877974298929629),\n",
       "   ('jupiter', 0.00430891535842575),\n",
       "   ('borland', 0.0041508194650061187),\n",
       "   ('het', 0.0038668267908505507),\n",
       "   ('intergraph', 0.0038058904617159187),\n",
       "   ('factory', 0.0035812197884953953),\n",
       "   ('paradox', 0.0034561576963550978),\n",
       "   ('captain', 0.0033253479803190158)]),\n",
       " (15,\n",
       "  [('militia', 0.0092128235982267426),\n",
       "   ('palestinian', 0.0059729164685874518),\n",
       "   ('cpr', 0.004686284364603395),\n",
       "   ('pts', 0.0040028517915765699),\n",
       "   ('handheld', 0.0038772150266758502),\n",
       "   ('sharks', 0.0038587387864929806),\n",
       "   ('igc', 0.0037003705150873848),\n",
       "   ('apc', 0.0034662736056090262),\n",
       "   ('jake', 0.0032698477461523233),\n",
       "   ('lehigh', 0.0031785274733683419)]),\n",
       " (16,\n",
       "  [('alaska', 0.0079605718735620869),\n",
       "   ('duke', 0.0051523030851937926),\n",
       "   ('col', 0.0048432462728980287),\n",
       "   ('russia', 0.0039719932178060328),\n",
       "   ('uoknor', 0.0039228543609895203),\n",
       "   ('aurora', 0.0038682315422029711),\n",
       "   ('princeton', 0.0038068285821953214),\n",
       "   ('nsmca', 0.0037177884777025499),\n",
       "   ('gene', 0.0036455849851698289),\n",
       "   ('stereo', 0.0034763208230052024)]),\n",
       " (17,\n",
       "  [('uuencode', 0.0072078249832380941),\n",
       "   ('msg', 0.0066826456843733029),\n",
       "   ('helmet', 0.0062624049825436741),\n",
       "   ('eos', 0.005927671329148035),\n",
       "   ('satan', 0.0058652634218759438),\n",
       "   ('dseg', 0.0057277629398535222),\n",
       "   ('homosexual', 0.0048206940386652034),\n",
       "   ('ics', 0.0041618254158387906),\n",
       "   ('gear', 0.0040704519881731701),\n",
       "   ('pyron', 0.0038082210285040204)]),\n",
       " (18,\n",
       "  [('entries', 0.017147934187645063),\n",
       "   ('myers', 0.016167472916643635),\n",
       "   ('x11r4', 0.007909293583366641),\n",
       "   ('radar', 0.0076424549665312994),\n",
       "   ('remark', 0.0075758144405586349),\n",
       "   ('cipher', 0.0075443974770796444),\n",
       "   ('maine', 0.0072329278698761565),\n",
       "   ('hamburg', 0.0068901221855506879),\n",
       "   ('senior', 0.0061756440454953125),\n",
       "   ('bontchev', 0.0061435301509686234)]),\n",
       " (19,\n",
       "  [('cubs', 0.0060873002860835773),\n",
       "   ('ufl', 0.0046822466862243761),\n",
       "   ('vitamin', 0.0043619487953724615),\n",
       "   ('temple', 0.0042737320124139333),\n",
       "   ('gsfc', 0.0039889260585051454),\n",
       "   ('mccall', 0.0034100500684956739),\n",
       "   ('astro', 0.0032781023207931331),\n",
       "   ('bellcore', 0.003267829832332651),\n",
       "   ('uranium', 0.0030868114552584855),\n",
       "   ('wesleyan', 0.0029836662482427201)])]"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 进行主题提取\n",
    "\n",
    "twp.topics_by_lda()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
